(ns metabase.query-processor.middleware.results-metadata
  "Middleware that stores metadata about results column types after running a query for a Card,
   and returns that metadata (which can be passed *back* to the backend when saving a Card) as well
   as a checksum in the API response."
  (:refer-clojure :exclude [mapv select-keys])
  (:require
   [clojure.string :as str]
   [malli.error :as me]
   [metabase.analyze.core :as analyze]
   [metabase.driver :as driver]
   [metabase.driver.util :as driver.u]
   [metabase.lib.metadata :as lib.metadata]
   [metabase.lib.schema :as lib.schema]
   [metabase.lib.schema.metadata :as lib.schema.metadata]
   [metabase.query-processor.reducible :as qp.reducible]
   [metabase.query-processor.schema :as qp.schema]
   ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
   [metabase.util :as u]
   [metabase.util.log :as log]
   [metabase.util.malli :as mu]
   [metabase.util.malli.registry :as mr]
   [metabase.util.performance :refer [mapv select-keys]]
   ^{:clj-kondo/ignore [:discouraged-namespace]}
   [toucan2.core :as t2]))

;;; +----------------------------------------------------------------------------------------------------------------+
;;; |                                                   Middleware                                                   |
;;; +----------------------------------------------------------------------------------------------------------------+

(mu/defn- comparable-metadata :- [:sequential :map]
  "Smooth out any unimportant differences in metadata so we can do an easy equality check."
  [metadata :- [:maybe [:sequential ::lib.schema.metadata/lib-or-legacy-column]]]
  (letfn [(remove-underscore-nil-keys
            ;; Sometimes we get an underscore version of a key with a nil value which is a duplicate of a key with a
            ;; dash. Remove the nil value
            [m]
            (reduce-kv
             (fn [acc k v]
               (let [kebab-cased-key (when (str/includes? (name k) "_")
                                       (keyword (str/replace (name k) "_" "-")))]
                 (if (and kebab-cased-key
                          (nil? v)
                          (contains? m kebab-cased-key))
                   acc
                   (assoc acc k v))))
             {} m))
          (standardize-metadata [m]
            (cond
              (keyword? m) (u/qualified-name m)
              (map? m) (-> (update-vals m standardize-metadata)
                           (dissoc :ident) ; `:ident` is deprecated and should no longer be present, but better safe than sorry.
                           (remove-underscore-nil-keys))
              (sequential? m) (mapv standardize-metadata m)
              (set? m) (into #{} (map standardize-metadata) m)
              :else m))]
    (mapv standardize-metadata metadata)))

(mu/defn- record-metadata!
  [{{:keys [card-id]} :info, :as query} :- ::lib.schema/query
   metadata                             :- [:maybe [:sequential ::lib.schema.metadata/lib-or-legacy-column]]]
  (try
    ;; At the very least we can skip the Extra DB call to update this Card's metadata results
    ;; if its DB doesn't support nested queries in the first place
    (let [actual-metadata (or (when-let [pivot-metadata (get-in query [:info :pivot/result-metadata])]
                                (when (sequential? pivot-metadata)
                                  pivot-metadata))
                              metadata)]
      (when (and actual-metadata
                 driver/*driver*
                 ;; pivot queries can run multiple queries, only record metadata for the main query
                 (not= actual-metadata :none)
                 (driver.u/supports? driver/*driver* :nested-queries (lib.metadata/database query))
                 card-id
                 ;; don't want to update metadata when we use a Card as a source Card.
                 (not (:qp/source-card-id query))
                 ;; Only update changed metadata
                 (not= (comparable-metadata actual-metadata)
                       (comparable-metadata
                        ;; existing usage -- don't use going forward
                        #_{:clj-kondo/ignore [:deprecated-var]}
                        (qp.store/miscellaneous-value [::card-stored-metadata]))))
        (when-let [error (me/humanize (mr/explain [:sequential ::lib.schema.metadata/lib-or-legacy-column] actual-metadata))]
          (throw (ex-info "Invalid result metadata!" {:error error, :metadata actual-metadata})))
        (t2/update! :model/Card card-id {:result_metadata actual-metadata
                                         :updated_at      :updated_at})))
    ;; if for some reason we weren't able to record results metadata for this query then just proceed as normal
    ;; rather than failing the entire query
    (catch Throwable e
      (log/error e "Error recording results metadata for query"))))

(defn- merge-final-column-metadata
  "Because insights are generated by reducing functions, they start working before the entire query metadata is in its
  final form. Some columns come back without type information, and thus get an initial base type of `:type/*` (unknown
  type); in this case, the `annotate` middleware scans the first few values and infers a base type, adding that
  information to the column metadata in the final result.

  This function merges inferred column base types added by `annotate` into the metadata generated by `insights`."
  [final-col-metadata insights-col-metadata]
  ;; the two metadatas will both be in order that matches the column order of the results
  (mapv
   (fn [{final-base-type      :base_type
         final-effective-type :effective_type
         :as final-col}
        {our-base-type      :base_type
         our-effective-type :effective_type
         :as insights-col}]
     (merge
      (select-keys final-col [:id :description :display_name :semantic_type :fk_target_field_id
                              :settings :field_ref :base_type :effective_type :database_type
                              :remapped_from :remapped_to :coercion_strategy :visibility_type
                              :was_binned :table_id])
      insights-col
      {:name (:name final-col)} ; The final cols have correctly disambiguated ID_2 names, but the insights cols don't.
      (when (= our-base-type :type/*)
        {:base_type final-base-type})
      (when (= our-effective-type :type/*)
        {:effective_type final-effective-type})))
   final-col-metadata
   insights-col-metadata))

(mu/defn- insights-xform :- fn?
  [orig-metadata :- [:maybe :map]
   record!       :- ifn?
   rf            :- ifn?]
  (qp.reducible/combine-additional-reducing-fns
   rf
   [(analyze/insights-rf orig-metadata)]
   (fn combine [result {:keys [metadata insights]}]
     (let [metadata (merge-final-column-metadata (-> result :data :cols) metadata)]
       (record! metadata)
       (rf (cond-> result
             (map? result)
             (update :data
                     assoc
                     ;; TODO: We agreed on the name `:result_metadata` everywhere, and this needs updating.
                     ;; It'll definitely break things on the FE, so a coordinated change is needed.
                     :results_metadata {:columns metadata}
                     :insights         insights)))))))

(mu/defn record-and-return-metadata! :- ::qp.schema/rff
  "Post-processing middleware that records metadata about the columns returned when running the query. Returns an rff."
  [{{:keys [skip-results-metadata?]} :middleware, :as query} :- ::qp.schema/any-query
   rff                                                       :- ::qp.schema/rff]
  (if skip-results-metadata?
    rff
    (let [record! (partial record-metadata! query)]
      (fn record-and-return-metadata!-rff* [metadata]
        (insights-xform metadata record! (rff metadata))))))

(mu/defn store-previous-result-metadata!
  "Store the previous value of a card's result metadata in the qp.store"
  [card :- [:maybe
            [:map
             [:result_metadata {:optional true} [:maybe [:sequential ::lib.schema.metadata/lib-or-legacy-column]]]]]]
  (when-let [result-metadata (:result_metadata card)]
    (if-let [error (me/humanize (mr/explain [:sequential ::lib.schema.metadata/lib-or-legacy-column] result-metadata))]
      (log/errorf "Invalid card result metadata, ignoring  it: %s" (pr-str error))
      ;; existing usage -- don't use going forward
      #_{:clj-kondo/ignore [:deprecated-var]}
      (qp.store/store-miscellaneous-value! [::card-stored-metadata] result-metadata))))
